"""
  Copyright (c) 2024 Northeastern University (China)
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
"""
# 任务：主要执行git clone命令下载项目
# 注意，只负责下载源码仓库

import hashlib
import math
import re
import subprocess
import os
import sys
import traceback
from concurrent.futures import ThreadPoolExecutor
from urllib.parse import urlparse
from tqdm import tqdm
from src.common_utils import configure
from src.common_utils.logger import logger
from src.common_utils.utils_file import read_jl, write_append_jl, write_jl, compress_dict

# 当前文件的路径
g_cur_dir = os.path.dirname(os.path.abspath(__file__))
# 当前文件的父路径
father_path = os.path.dirname(g_cur_dir)
sys.path.append(father_path)
class Download:
    # alloss_path = g_cur_dir + r'\data_out\process_oss.jl'  # 所有需要下载的oss的名字和地址
    alloss_path = os.path.join(configure.CONFIG['ossdb_construct'],'process_oss.jl') # 所有需要下载的oss的名字和地址
    # success_oss_path=g_cur_dir+ r'\data_out\success_clone_oss.jl'
    success_oss_path = os.path.join(configure.CONFIG['ossdb_construct'],'success_clone_oss.jl')
    down_path=configure.CONFIG['dataset_repo']#源码存放处
    # all_cloned_oss_path = g_cur_dir + r'\data_out\cloned_oss.jl'#这个用来下一步生成签名读的文件,所有已经成功下载的本地仓库还有信息
    # cloned_projects_path=g_cur_dir + r'\data_out\cloned_projects_paths.jl'  # 仅记录本次下载成功的项目路径，下一次扫描依赖项仅扫描这些项目
    all_cloned_oss_path = os.path.join(configure.CONFIG['ossdb_construct'],'cloned_oss.jl')   # 这个用来下一步生成签名读的文件,所有已经成功下载的本地仓库还有信息
    cloned_projects_path = os.path.join(configure.CONFIG['ossdb_construct'],'cloned_projects_paths.jl')  # 仅记录本次下载成功的项目路径，下一次扫描依赖项仅扫描这些项目
    all_cloned_oss=[]
    def __init__(self):
        all_oss = read_jl(self.alloss_path)
        self.exist_oss = read_jl(self.success_oss_path)
        self.process_oss = [item for item in all_oss if item not in self.exist_oss]
        self.all_cloned_oss=read_jl(self.all_cloned_oss_path)
        if os.path.exists(self.cloned_projects_path):
            open(self.cloned_projects_path, 'w').close()

    def clean_url(self,urlstr):
        # 去除网址中间的多余斜杠
        clean_url = re.sub(r"(?<!:)/{2,}", "/", urlstr)
        clean_url = clean_url.strip()  # 去掉空格
        # 先去除字符串中的转义字符
        parsed_url = urlparse(clean_url)
        clean_url = parsed_url.geturl()
        # 去除网址末尾的多余斜杠
        if clean_url.endswith("/"):
            clean_url = clean_url.rstrip("/")
        if clean_url.endswith(".git"):
            clean_url = clean_url[:-4]  # 去除末尾的 ".git"，切片操作
        return clean_url.lower()  # 认为主流网站对大小写不敏感
    def clone_repo(self,_idx,list_arr):
        """
        克隆并生成oss的文件
        :return:
        """
        for oss in  tqdm(list_arr,total=len(list_arr),desc=f"thread {_idx}oss下载进度", unit="个oss"):#待处理的oss项目总数百分比显示
                try:
                    oss_url=self.clean_url(oss['url'])
                    repo_name = oss_url.split('/')[-1].replace(".git", "")
                    repo_url_hash=hashlib.md5(oss_url.encode('utf-8')).hexdigest()
                    repo_full_name=repo_name+"_"+repo_url_hash#为了避免出现同名但是非同一个软件包不下载的情况，把url哈希，选用md5是因为这个长度适中
                    repo_path = os.path.join(self.down_path, repo_full_name)
                    data=None
                    if not os.path.exists(repo_path):
                        #执行git clone 命令下载源码
                        githubtoken=configure.CONFIG['githubtoken']
                        gitlabtoken = configure.CONFIG['gitlabtoken']
                        # 构建带有访问令牌的 oss_url
                        if "https://github.com" in oss_url:
                            url_with_token = f"https://{githubtoken}@{oss_url[8:]}"
                            clone_command = "git clone " + url_with_token + " " + repo_path
                        elif "https://gitlab.com" in oss_url:
                            url_with_token = f"https://oauth2:{gitlabtoken}@{oss_url[8:]}"
                            clone_command = "git clone " + url_with_token + " " + repo_path
                        else:
                            clone_command = "git clone " + oss_url + " " + repo_path
                        clone_result = subprocess.check_output(
                            clone_command, stderr=subprocess.STDOUT, shell=True
                        ).decode()
                        print(f"clone command result:{clone_result}")
                        data={
                            'oss_name':oss['name'],
                            'oss_url':oss['url'],
                            'source':oss['source'],
                            'repo_full_name':repo_full_name,
                            'repo_path':repo_path
                        }
                    else:
                        print(repo_path)
                    if data is not None and data not in self.all_cloned_oss :
                        write_append_jl(self.all_cloned_oss_path,data)
                    if oss not in self.exist_oss:
                        write_append_jl(self.success_oss_path,oss)
                        self.exist_oss.append(oss)
                    write_append_jl(self.cloned_projects_path, repo_path)
                except Exception as e:
                    print("clone_extract_signatures error:", e)
                    logger.error('[+]  An error occurred', exc_info=True)
                    traceback.print_exc()
                    continue

    def split_list(self, list, n):
        """
        将数据n等分，分发给n个线程执行

        :param list: 原始完整list
        :param n: 切分数量，即线程数量
        :return: 切分后的list起始位置
        """
        step_count = math.ceil(len(list) / n)
        for i in range(0, len(list), step_count):
            end = i + step_count
            if end > len(list):
                end = len(list)
            # print(f"yield: {i}:{end}")
            yield list[i:end]

    def main_multithread(self, num_threads,clone_osses=None):
        if clone_osses is None:
            clone_osses=self.process_oss
        if len(clone_osses) != 0:
            print(f"共计处理:{len(clone_osses)}")
            with ThreadPoolExecutor(max_workers=num_threads) as executor:
                for idx, list_arr in enumerate(self.split_list(clone_osses, num_threads)):
                    executor.submit(self.clone_repo, idx + 1, list_arr)
        else:
            print("没有需要处理的 OSS")

if __name__ == "__main__":
    download=Download()
    download.main_multithread(num_threads=4)

